import numpy as np
from Routes import *
from pre_processing import *
from sklearn.preprocessing import MinMaxScaler
from sklearn.decomposition import PCA

succ='_part1'
routes=Routes()
readRoutes(routes)
linklist=routes.get_links()
eps=0.001
kk=1
limit=5
wea=30
count=32

def data_clean(datas,labels,tests):
    d=[]
    l=[]
    dels=[]
    lt=len(datas)
    lv=len(datas[0])-7
    lv2=len(labels[0])
    s=[0]*100
    tot=[0]*100
    aver=[0.0]*100
    aver2=[0.0]*100

    for i in range(0,lv):
        ss=0
        for j in range(0,lt):
            if datas[j][i]>eps:
                aver[i]+=datas[j][i]
                ss+=1
        aver[i]/=ss
    for i in range(0,lv2):
        ss=0
        for j in range(0,lt):
            if labels[j][i]>eps:
                aver2[i]+=labels[j][i]
                ss+=1
        aver2[i]/=ss
    for i in range(0,lt):
        t=0
        for j in range(0,lv):
            if datas[i][j]<eps and j<lv-wea:
                t+=1
                continue
#            if i>0 and i<lt-1:
#                if datas[i-1][j]>eps and datas[i+1][j]>eps:
#                    if abs(datas[i][j]-datas[i-1][j])>aver[j]*kk and abs(datas[i][j]-datas[i+1][j])>aver[j]*kk:
#                        datas[i][j]=(datas[i-1][j]+datas[i+1][j])/2
#                        t+=1
#                        continue
            if datas[i][j]>aver[j]*limit:
                datas[i][j]=aver[j]*limit
                t+=1
        for j in range(0,lv2):
            if labels[i][j]<eps:
                t+=1
                continue
#            if i>0 and i<lt-1:
#                if labels[i-1][j]>eps and labels[i+1][j]>eps:
#                    if abs(labels[i][j]-labels[i-1][j])>aver2[j]*kk and abs(labels[i][j]-labels[i+1][j])>aver2[j]*kk:
#                        labels[i][j]=(labels[i-1][j]+labels[i+1][j])/2
#                        t+=1
#                        continue
            if labels[i][j]>aver2[j]*limit:
                labels[i][j]=aver2[j]*limit
                t+=1
        s[i]=t
        tot[t]+=1

    k=100
    for i in range(1,100):
        tot[i]+=tot[i-1]
        if tot[i]>count:
            k=i
            break
    for i in range(0,lt):
        if s[i]<=k or i>=lt-7:
            dd=datas[i]
            ll=labels[i]
            for j in range(0,lv):
                if dd[j]<eps and j<lv-wea:
                    ave=0.0
                    ss=0
                    for ii in range(i-4,i+5):
                        if ii>=0 and ii<lt and datas[ii][j]>eps:
                            ave+=datas[ii][j]
                            ss+=1
                    if ss==0:
                        ave=aver[j]
                    else:
                        ave/=ss
                    dd[j]=ave
            for j in range(0,lv2):
                if ll[j]<eps:
                    ave=0.0
                    ss=0
                    for ii in range(i-4,i+5):
                        if ii>=0 and ii<lt and labels[ii][j]>eps:
                            ave+=labels[ii][j]
                            ss+=1
                    if ss==0:
                        ave=aver2[j]
                    else:
                        ave/=ss
                    ll[j]=ave
            d.append(dd)
            l.append(ll)
    for i in range(0,lv):
        for j in range(0,len(tests)):
            if tests[j][i]<eps and i<lv-wea:
                tests[j][i]=aver[i]
            if tests[j][i]>=aver[i]*limit:
                tests[j][i]=aver[i]*limit

    print 'days left:',len(d)
    return (d,l,tests)

for i in linklist:
    name='data'+str(i)+succ
    target=name+'_modified'
    traindata=np.load('./data_links/train'+name+'.npz')
    testdata=np.load('./data_links/test'+name+'.npz')
    datas=traindata['datas']
    labels=traindata['labels']
    tests=testdata['tests']
    (datas,labels,tests)=data_clean(datas,labels,tests)
    scaler=MinMaxScaler()
    datas=scaler.fit_transform(datas)
    tests=scaler.transform(tests)
    np.savez('./data_links/train'+target,datas=np.array(datas),labels=np.array(labels))
    np.savez('./data_links/test'+target,tests=np.array(tests))
    #pca=PCA(n_components=25)
    #datas=pca.fit_transform(datas)
    #tests=pca.transform(tests)
